import urlparse, logging, os, functools

logger = logging.getLogger("curlspider")

def is_valid_url(url):
   "check form & whether top-level domain is correct"
   domain = urlparse.urlparse(url).hostname
   return False if not url.startswith("http://") or domain.split('.')[-1] not in TLDS else True


def validate_urls(urls):
   "filter out bad urls"
   invalids = []
   for url in urls:
      if not is_valid_url(url):
         logger.warn("ignoring invalid url %s" % url)
         invalids.append(url)
   return [str(url) for url in urls if url not in invalids]


def get_proxywriter(writer, init):
   "proxy writer utility for the spider callback writer"
   partial = functools.partial(writer, init)

   def proxy(buf, func=partial):
      return func(buf)

   return proxy


def debug_initializer(original):
   logger.debug("START %s" % original)
   return original

def debug_finalizer(original, effective):
   if effective == original:
      logger.debug("OK %s" % original)
   else:
      logger.debug("OK %s > %s" % (original, effective))


def debug_errhandler(original, effective, errno, errmsg):
   if effective == original:
      logger.warn("FAILED %s: %s %s" % (original, str(errno), errmsg))
   else:
      logger.warn("FAILED %s > %s: %s %s" % (original, effective, str(errno), errmsg))


def file_initializer(url, pathgenerator=None, overwrite=False):
   "get callbacks for WRITEFUNCTION and a finalizer that gets called upon URL completion"

   defaultgenerator = lambda u: u[7:] if u.startswith("http://") else u
   generator = pathgenerator or defaultgenerator
   filepath = generator(url)
   if os.path.exists(filepath) and not overwrite:
      raise Exception("file exists: set 'overwrite' to True to override")
   folder = os.sep.join(filepath.split('/')[:-1])

   if not os.path.isdir(folder):
      os.makedirs(folder)

   return (open(filepath, "wb"), url)


def file_writer((fileobj, original), buf):
   "write retrieved files"
   if len(buf.strip()) > 0:
      try:
         fileobj.write(buf)
         fileobj.flush()
      except Exception, e:
         logger.error("callback: %s: could not write to %s: %s" % (original, path, e))
   else:
      reason = "empty buffer" if len(buf.strip())==0 else "already exists"
      logger.debug("callback: not writing %s: %s" % (original, reason))


def file_finalizer((fileobj, original), effective):
   "close files etc"
   fileobj.close()


def file_errhandler((fileobj, original), effective, errno, errmsg):
   fileobj.close()
   os.remove(fileobj.name)
   if original == effective:
      logger.warn("error: %s (%s %s) - closed and removed %s" % (original, errno, errmsg, fileobj.name))
   else:
      logger.warn("error: %s > %s (%s %s) - closed and removed %s" % (original, effective, errno, errmsg, fileobj.name))


# top-level domains for url checking
TLDS = (
   "aero", "cat", "com", "coop", "edu", "gov", "info", "int", "jobs",
   "mil", "mobi", "museum", "name", "net", "org", "pro", "tel", "travel",
   "ac", "ad", "ae", "af", "ag", "ai", "al", "am", "an", "ao", "aq", "ar", "as", "at", "au", "aw", "ax", "az",
   "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "br", "bs", "bt", "bv", "bw", "by", "bz",
   "ca", "cc", "cd", "cf", "cg", "ch", "ci", "ck", "cl", "cm", "cn", "co", "cr", "cu", "cv", "cx", "cy", "cz",
   "de", "dj", "dk", "dm", "do", "dz",
   "ec", "ee", "eg", "er", "es", "et", "eu",
   "fi", "fj", "fk", "fm", "fo", "fr",
   "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gi", "gl", "gm", "gn", "gp", "gq", "gr", "gs", "gt", "gu", "gw", "gy",
   "hk", "hm", "hn", "hr", "ht", "hu",
   "id", "ie", "il", "im", "in", "io", "iq", "ir", "is", "it",
   "je", "jm", "jo", "jp",
   "ke", "kg", "kh", "ki", "km", "kn", "kp", "kr", "kw", "ky", "kz",
   "la", "lb", "lc", "li", "lk", "lr", "ls", "lt", "lu", "lv", "ly",
   "ma", "mc", "md", "me", "mg", "mh", "mk", "ml", "mm", "mn", "mo", "mp", "mq", "mr", "ms", "mt", "mu", "mv", "mw", "mx", "my", "mz",
   "na", "nc", "ne", "nf", "ng", "ni", "nl", "no", "np", "nr", "nu", "nz",
   "om",
   "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pm", "pn", "pr", "ps", "pt", "pw", "py",
   "qa",
   "re", "ro", "rs", "ru", "rw",
   "sa", "sb", "sc", "sd", "se", "sg", "sh", "si", "sj", "sk", "sl", "sm", "sn", "so", "sr", "st", "su", "sv", "sy", "sz",
   "tc", "td", "tf", "tg", "th", "tj", "tk", "tl", "tm", "tn", "to", "tp", "tr", "tt", "tv", "tw", "tz",
   "ua", "ug", "uk", "um", "us", "uy", "uz",
   "va", "vc", "ve", "vg", "vi", "vn", "vu",
   "wf", "ws",
   "ye", "yt", "yu",
   "za", "zm", "zw"
)
